package util;

import java.io.IOException;
import java.net.SocketException;
import java.util.Iterator;
import java.util.List;
import java.util.regex.Matcher;
import java.util.regex.Pattern;

import org.apache.http.client.params.CookiePolicy;
import org.jdom.Document;
import org.jdom.Element;
import org.jdom.JDOMException;
import org.jdom.input.SAXBuilder;

import com.gargoylesoftware.htmlunit.BrowserVersion;
import com.gargoylesoftware.htmlunit.FailingHttpStatusCodeException;
import com.gargoylesoftware.htmlunit.ProxyConfig;
import com.gargoylesoftware.htmlunit.WebClient;
import com.gargoylesoftware.htmlunit.html.HtmlPage;
import com.gargoylesoftware.htmlunit.util.Cookie;

public class SpiderBodyXML {
	// 抓取失败后多次重启，每次重启间隔的增量
	final int errorSleepIncrement = 30000;
	final int interval = 30000;
	HtmlPage spiderPage = null;
	WebClient webClient = null;
	String bodyString = null;

	public synchronized String getWebSiteBodyString() {
		String bodyString = this.bodyString;
		//System.out.println(bodyString);
		return bodyString;
	}

	public void clear() {
		webClient.closeAllWindows();
		webClient.getCookieManager().clearCookies();
	}

	public void SpiderPageToXML(String site) {
		System.out.println(site);
		// 这个循环的目的是在出现错误是，可以多次重复抓取
		for (int reConectTimes = 0; reConectTimes < repeatTimes;) {
			
			try {
				// sougou是直接封，如果被封下一句就会出异常，跳到catch块
				System.out.println("网页抓取开始");
				this.spiderPage = webClient.getPage(site);
				System.out.println("网页抓取结束");
				this.bodyString = spiderPage.getBody().asXml();
				// 抓取成功后直接退出循环
				reConectTimes = repeatTimes;
			}
			// 这个是在Google遇到屏蔽关键字时出现的Exception
			catch (SocketException e) {

				this.bodyString = "0";
				try {
					System.out.println("-----SocketException-----");
					Thread.sleep(120000);
					continue;
				} catch (InterruptedException e1) {
					System.out.println("-----InterruptedException-----");
					e1.printStackTrace();
				}

			} catch (IOException e) {
				// 出错链连接错误，连续几次，如果连续连接最多次数title写成0000、content写成0000，否则暂停10秒继续连接
				if (reConectTimes == repeatTimes - 1) {
					this.bodyString = "0";
					break;
				} else
					try {
						// 出错后且没到错多出错次数，会延时一段继续执行
						System.out.println(" IOException FailingHttpStatusCodeException now it will spider again");
						Thread.sleep(interval + errorSleepIncrement * reConectTimes);
						reConectTimes++;
						continue;
					} catch (InterruptedException e1) {
						System.out.println("-----InterruptedException002-----");
						e1.printStackTrace();
					}
			}
			// FailingHttpStatusCodeException,这里抓runtimeException，youdao会报出错误的连接状态不抓取的话会跳出程序造成中断，这里抓一下出现错误状态就停5分钟重连
			catch (FailingHttpStatusCodeException e) {

				System.out.println("FailingHttpStatusCodeException now it will spider again");
				try {
					Thread.sleep(300000);
					reConectTimes++;
					continue;
				} catch (InterruptedException e1) {
					// TODO Auto-generated catch block
					e1.printStackTrace();
				}

			} finally {
				webClient.closeAllWindows();
				webClient.getCookieManager().clearCookies();

			}

		}
	}

	int repeatTimes = 0;

	public void setRepeatTimes(int repeatTimes) {
		this.repeatTimes = repeatTimes;
	}
	
	SpiderBodyXML(){
		String prcString = null;
		webClient = new WebClient(BrowserVersion.FIREFOX_3_6);
		webClient.setTimeout(50000);
		System.setProperty("apache.commons.httpclient.cookiespec", CookiePolicy.BROWSER_COMPATIBILITY);
		webClient.setJavaScriptEnabled(false);
		// 设置cookies
		SAXBuilder builder = new SAXBuilder(false);
		List sourceSiteParameterList = null;
		Document doc;
		boolean cookiesTag = false;
		try {
			doc = builder.build("urlSpider.xml");
			Element parameter = doc.getRootElement();
			sourceSiteParameterList = parameter.getChildren("sourceSite");
			for (Iterator iter = sourceSiteParameterList.iterator(); iter.hasNext();) {
				Element sourceSiteParameter = (Element) iter.next();
				cookiesTag = new Boolean(sourceSiteParameter.getChildTextTrim("cookiesTag")).booleanValue();
				prcString = sourceSiteParameter.getChildTextTrim("proxy");
				if (cookiesTag) {
					List cookiesList = sourceSiteParameter.getChildren("cookies");
					for (Iterator itercookies = cookiesList.iterator(); itercookies.hasNext();) {
						Element cookiesParameter = (Element) itercookies.next();
						String cookiesName = cookiesParameter.getChildTextTrim("cookiesName");
						String cookiesValue = cookiesParameter.getChildTextTrim("cookiesValue");
						String cookiesDomain = cookiesParameter.getChildTextTrim("cookiesDomain");
						webClient.getCookieManager().addCookie(new Cookie(cookiesDomain, cookiesName, cookiesValue));
					}
				}
			}
		} catch (JDOMException e) {
			// TODO Auto-generated catch block
			e.printStackTrace();
		} catch (IOException e) {
			// TODO Auto-generated catch block
			e.printStackTrace();
		}
		System.out.println(prcString);
		// 里的代理地址
		Pattern p = Pattern.compile("(.*)(:)(.*)");
		Matcher m = p.matcher(prcString);
		if (m.find()) {
			ProxyConfig prc = new ProxyConfig(m.group(1), new Integer(m.group(3)).intValue());
			webClient.setProxyConfig(prc);
		}
	}
}
